package be.destin.harvesting.simpler;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Collection;
import java.util.Date;
import java.util.HashSet;

import org.apache.log4j.Logger;

import be.destin.beans.Reflector;
import be.destin.skos.core.Concept;
import be.destin.skos.core.ConceptScheme;
import be.destin.skos.core.Count;
import be.destin.skos.core.NoScheme;
import be.destin.skos.core.SchemeUsage;
import be.destin.skos.core.SchemeUsageList;
import be.destin.skos.core.SkosManager;
import be.destin.skos.core.TermList;
import be.destin.skos.html.SkosHtmlManager;

public class UsageHarvester {
	/** log4j category */
	private static Logger log = Logger.getLogger(UsageHarvester.class);

	public UsageHarvester(SkosHtmlManager skosHtmlManager, String application) throws Exception {
		String applicationRoot = skosHtmlManager.getApplicationRoot(application);
		SkosManager skosManager = skosHtmlManager.getSkosManager();
		String configDir = skosManager.getConfigDirectory();

		SchemeUsageList usageList = skosHtmlManager.getSkosManager().getHarvestedUsage(application);
		if (usageList == null || usageList.isEmpty()) return;

		int nbURL = 0;
		boolean fatalError = false;
		String message = "";

		File cacheFile = new File(configDir+File.separator+application+File.separator+"crawlinprogress.txt");

		BufferedReader cacheReader = null;
		boolean createCache = false;

		HashSet<String> harvested = new HashSet<String>();
		try {
			cacheReader = new BufferedReader(new FileReader(cacheFile));
			String strLine;
			while ((strLine = cacheReader.readLine()) != null)   {
				String[] fields = strLine.split("\t");
				if (fields == null || fields.length < 3) {
					log.error("line too short (Concept tab Role tab Count are requested) in "+cacheFile.getPath());
					break; // too few fields: BAD FILE!
				}
				int count = 0;
				if (!fields[2].isEmpty()) {
					try {
						count = Integer.parseInt(fields[2]);
					} catch (NumberFormatException e) {
						log.error("invalid count \""+fields[2]+"\" in "+cacheFile.getPath());
						break; // BAD NUMBER
					}
				}
				String role = fields[1];
				Concept aRole = skosManager.getRoles().getConcept(role);
				if (aRole == null) {
					log.info("Role "+role+" is in "+cacheFile.getPath()+" but not in the list of defined roles. Line ignored.");
					continue;
				}
				Concept aConcept = skosManager.getScheme_Concept(fields[0]);
				if (aConcept == null) {
					log.info("Concept "+fields[0]+" is in "+cacheFile.getPath()+" but not in the known Concepts. Line ignored.");
					continue;
				}
				harvested.add(aConcept.getScheme_About()+'\t'+role);
				aConcept.putApplicationUsage(application, role, count, null, true);
				ConceptScheme aScheme = aConcept.getInScheme();
				if (aScheme != null) {
					SchemeUsage su = aScheme.getApplicationUsage(application, role);
					if (su == null) {
						su = new SchemeUsage(aConcept.getInScheme(),application, role);
						aScheme.putSchemeUsage(su);
					}
				}
			}
			//Close the input stream
			cacheReader.close();
		} catch (Exception e){//Catch exception if any
			log.error(cacheFile.getPath() + ": " + e.getMessage());
			if (cacheReader != null) {
				try {
					cacheReader.close();
				} catch (IOException e1) {
					// TODO Auto-generated catch block
					e1.printStackTrace();
				}
			}
			createCache = true;
		}

		BufferedWriter cacheWriter = null;
		try {
			cacheWriter = new BufferedWriter(new FileWriter(cacheFile, !createCache));
		} catch (IOException ioe) {
			log.error(cacheFile.getAbsolutePath(),ioe);
			try {
				if (cacheWriter != null) cacheWriter.close();
			} catch (IOException ioe2) {
			}
			return; // DO NOT CONTINUE!
		}
		long totalResponseTime = 0;
		for (SchemeUsage aUsage : usageList) {
			if (fatalError) break;
			log.info("Harvesting for scheme "+aUsage.getInScheme().getAbout()+", application="+application+", role="+aUsage.getRole());
			String harvestTemplate = aUsage.getHarvestURL();
			String harvestConditionTemplate = aUsage.getHarvestCondition();
			if (harvestConditionTemplate != null) harvestConditionTemplate = harvestConditionTemplate.trim();
			Object[] beans = new Object[2];
			NoScheme currScheme = aUsage.getInScheme();
			if (currScheme != null) {
				Collection<Concept> toQuery = currScheme.getAllConcepts();
				if (toQuery != null) {
					for (Concept aConcept : toQuery) {
						
						if (harvested.contains(aConcept.getScheme_About()+'\t'+aUsage.getRole())) continue;

						beans[0] = aConcept;
						beans[1] = aConcept.getInScheme();
						String condition = null;
						if (harvestConditionTemplate == null || harvestConditionTemplate.isEmpty()) {
							condition = "OK";
						} else {
							condition = Reflector.substitute(aConcept,harvestConditionTemplate,TermList.anyLanguage).trim();
						}
						if (condition != null && !condition.isEmpty()) {
							String url = Reflector.substituteURL(applicationRoot,beans,harvestTemplate,TermList.anyLanguage);
							URL sourceURL = new URL(url);
							BufferedReader in = null;
							String rawData = null;
							long begin = System.currentTimeMillis();
							HttpURLConnection inConnection = null;
							InputStream inStream = null;
							int httpStatus = 0;
							nbURL ++;
							try {
								inConnection = (HttpURLConnection) sourceURL.openConnection();
								inStream = inConnection.getInputStream();
								in = new BufferedReader(new InputStreamReader(inStream));
								StringBuffer sb = new StringBuffer();
								String str;
								while ((str = in.readLine()) != null) {
									sb.append(str); 
								}
								rawData = sb.toString();
								httpStatus = inConnection.getResponseCode();
							} catch (java.net.SocketException soe) {
								log.error(url, soe);
								fatalError = true;
							} catch (IOException ioe) {
								log.error(url, ioe);
								if (inConnection != null) {
									httpStatus = inConnection.getResponseCode();
									fatalError = httpStatus != 404;
								}
							} finally {
								if (in != null) in.close();
							}
							long duration = System.currentTimeMillis() - begin;
							totalResponseTime += duration;
							Thread.sleep( duration*2 ); // To be nice pause for response time between transactions
							if (rawData != null && !fatalError) {
								boolean failed = aUsage.harvestFailed(rawData);
								try {
									if (!failed) {
										Count harvestCount = aUsage.harvestCount(aConcept, rawData);
										if (harvestCount != null) {
											aConcept.putApplicationUsage(aUsage.getApplication(), harvestCount.getRole(),
													harvestCount.getCount(), harvestCount.getCategory(), harvestCount.isPure());
											if (cacheWriter != null) {
												//TODO: category and "pure" are not written for now but could be added to then end of the line
												message = aConcept.getScheme_About()+'\t'+harvestCount.getRole()+'\t'+harvestCount.getCount();
												log.info(nbURL+": "+message);
											}
										} else {
											log.info(nbURL+": "+aConcept.getScheme_About()+": NO COUNT");
											message = aConcept.getScheme_About()+'\t'+aUsage.getRole()+"\t0";
										}
									} else {
										log.info(nbURL+": "+aConcept.getScheme_About()+": FAILED");
										message = aConcept.getScheme_About()+'\t'+aUsage.getRole()+"\t0";
									}
									cacheWriter.write(message);
									cacheWriter.newLine();
									cacheWriter.flush();
								} catch (IOException e) {
									log.error( nbURL+": "+e.toString() );
									fatalError = true;
								}
							}
						}
					}
				}
			}
		}
		cacheWriter.close();
		if (!fatalError) {
			log.info(nbURL+" URLs exploration ends without error.");
			// (1) get today's date
			Date today = Calendar.getInstance().getTime();

			// (2) create our date "formatter" (the date format we want)
			SimpleDateFormat formatter = new SimpleDateFormat("yyyyMMdd-HHmmss");

			// (3) create a new String using the date format we want
			String fileName = formatter.format(today);
			// e.g.20090906-082323"
			File dest  = new File(configDir+File.separator+"usage"+File.separator+application+fileName+".txt");
			cacheFile.renameTo(dest);
		} else {
			log.error(nbURL+" URLs ends following server or network error.");
		}
		if (nbURL > 0) {
			log.info("Mean response time="+(totalResponseTime / nbURL)+" ms");
		}
	}
}
